package extractorCode;

import java.io.BufferedInputStream;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import java.io.*;

public class link1Text {
	
	private final static int INITIAL_BUF_SIZE = 65536;

	public static void main(String args[]) throws Exception
	{
				
		int count=0;
		String url;
	
		 BufferedReader reader= new BufferedReader(new FileReader("c:\\keywords-link1-edited.txt"));

		    String keyword;
		    int lastIndex;
		    while((keyword=reader.readLine())!=null)
		    {
		    	
		    	String[] splitarr=keyword.split("\t");
		    	String key = splitarr[0];
		    	int maxPage=Integer.parseInt(splitarr[1]);
		    	System.out.println(keyword);
		       if(maxPage>-1)
		       {
		    	
			
			String urlString;
			BufferedReader br = new BufferedReader(new FileReader("C:\\Users\\Anu\\Desktop\\Website1\\link1-links\\link1-"+key+".txt"));

			count=0;
		    int prevCount=0;
		while((urlString=br.readLine())!=null)
		{	
		System.out.println(urlString);
		String[] splitarr2= urlString.split("\t");
		 url= splitarr2[0];
			url=url.replace("href=","");
			url=url.replace("\"","");
		try
		{

		String page= (fetchPage("http://nasional.kompas.com/read/2008/06/03/17595938/ormas-ormas.nu.tuntut.pembubaran.fpi"));
//		while(page!="")
//		{
//			Thread.sleep(10000);
//			 page = (fetchPage(url));
//		}
		System.out.println(page);
		page= page.replaceAll(">\\s*<", "><");
		
		page = page.replaceAll("\t", "");
		page = page.replaceAll("\b", "");
		page = page.replaceAll("\r", "");
		page = page.replaceAll("\n", "");
		page = page.replaceAll("\r\n", "");
		/*if(!page.equals(""))
		{
			
		BufferedWriter wrPage = new BufferedWriter( new FileWriter("C:\\newData2\\Website1\\link1-"+key+"-"+(count++)+".txt"));
		prevCount=count;
		
		wrPage.write(page);
		wrPage.close();
		
		}*/
		Matcher newsItem = Pattern.compile("<div id=\"article_body\">(.*?)</div>", Pattern.CASE_INSENSITIVE).matcher( page );
		
		Matcher newsItem2 = Pattern.compile("<div class=\"content_kiri_detail\">(.*?)<div class=\"content_kanan\">", Pattern.CASE_INSENSITIVE).matcher( page );
		
		File f = new File("c:\\newData2\\Website1\\link1-"+key);
		try{
		f.mkdir();
		}
		catch(Exception e)
		{}
		while(newsItem2.find())
		{
			System.out.println(count);
		   String text="";
			//System.out.println(newsItem.group(1));
			//if(newsItem.find())
		    text = newsItem2.group(1);
			//else
				//if(newsItem2.find())
				//text=newsItem2.group(1);
		System.out.println(text);
		text = text.replaceAll("<br />","\r\n");
		text = text.replaceAll("\\<.*?>","");
		text = text.replaceAll("[^\\p{ASCII}]", "");
		
		//BufferedWriter wr = new BufferedWriter( new FileWriter("C:\\newData2\\Website1\\link1-"+key+"\\link1-"+key+"-"+(count++)+".txt"));
		//wr.write(text);
		//wr.close();
		
		//System.out.println(text);
	//	writer.write("http://newsbusters.org"+newsItem.group(1)+"\n");
		}
		
		while(newsItem.find())
		{
			System.out.println(count);
		   String text="";
			//System.out.println(newsItem.group(1));
			//if(newsItem.find())
		    text = newsItem.group(1);
			//else
				//if(newsItem2.find())
				//text=newsItem2.group(1);
		System.out.println(text);
		text = text.replaceAll("<br />","\r\n");
		text = text.replaceAll("\\<.*?>","");
		text = text.replaceAll("[^\\p{ASCII}]", "");
	
		//BufferedWriter wr = new BufferedWriter( new FileWriter("C:\\newData2\\Website1\\link1-"+key+"\\link1-"+key+"-"+(count++)+".txt"));
		//wr.write(text);
		//wr.close();
		
		//System.out.println(text);
	//	writer.write("http://newsbusters.org"+newsItem.group(1)+"\n");
		}
		 if(count==prevCount)count++;
		}
		catch(Exception e)
		{
			System.out.println(e);
		}
		
		}
		
		br.close();
		}}

	}
	
	
	
	private static String fetchPage(String urlString){
		StringBuilder buffer = new StringBuilder(INITIAL_BUF_SIZE);
		try {
			URL url = new URL(urlString);
			
			  HttpURLConnection.setFollowRedirects(false);
              HttpURLConnection urlConn = (HttpURLConnection)url.openConnection();
               System.out.println("Connection made");
               urlConn.setRequestProperty("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
               /*urlConn.setRequestProperty("Accept-Charset","ISO-8859-1,utf-8;q=0.7,*;q=0.3");
               urlConn.setRequestProperty("Accept-Encoding","gzip,deflate,sdch");
               urlConn.setRequestProperty("Accept-Language","en-US,en;q=0.8");
               */urlConn.setRequestProperty("Cache-Control","max-age=0");
               urlConn.setRequestProperty("Connection","keep-alive");
             //  urlConn.setRequestProperty("Cookie","__utma=194028855.479991770.1322019053.1322019053.1322021581.2; __utmz=194028855.1322019053.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none); BIGipServerwww.gip.kompas.co.id_pool=319631882.20480.0000; K_SID=a%3A4%3A%7Bs%3A10%3A%22session_id%22%3Bs%3A32%3A%22e46c99996f3ccb613f41d8587f3c9da6%22%3Bs%3A10%3A%22ip_address%22%3Bs%3A12%3A%2268.99.87.243%22%3Bs%3A10%3A%22user_agent%22%3Bs%3A50%3A%22Mozilla%2F5.0+%28Windows+NT+6.1%3B+WOW64%29+AppleWebKit%2F53%22%3Bs%3A13%3A%22last_activity%22%3Bi%3A1323474883%3B%7D12da1690c9e05cb1bc645efd9d64e639; populertabs=0; __auc=d86f1336132ef64fd1583fbeeed; __asc=b26b2b7c134250acf8ab6838bdc; __utma=44297659.1496229451.1322019850.1322019850.1323471390.2; __utmb=44297659.10.10.1323471393; __utmc=44297659; __utmz=44297659.1322019854.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none); _em_hl=1; _em_vt=c75635124ce99a65accbd08666fb4e67d01bc942a0-798213234ee29fc9; _em_v=fcd56f659d7822459cd89e4cb6034ee292228b5c79-511006554ee29fc9");
               urlConn.setRequestProperty("Host","nasional.kompas.com");
               //urlConn.setRequestProperty("Referer","http://nasional.kompas.com/read/2010/05/22/16440487/Perangi.Trafficking.Polisi.Masyarakat.Dibentuk");
               urlConn.setRequestProperty("User-Agent","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/15.0.874.121 Safari/535.2");
       		
              //urlConn.setRequestProperty("User-agent","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/15.0.874.121 Safari/535.2");
              urlConn.connect();
              BufferedReader reader = new BufferedReader(new InputStreamReader(urlConn.getInputStream()));

			//BufferedInputStream reader = new BufferedInputStream(url.openStream());
			int temp = reader.read();
			while(temp != -1){
				buffer.append((char)temp);
				temp = reader.read();
			}
		} catch (MalformedURLException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		System.out.println();
		return buffer.toString();
	}

}
